/***************************************************************
*  Copyright (c) 2007 by GroupMe! Team (www.groupme.net)
*  All rights reserved.
*
*  This file is part of the GroupMe! Project. Source code of 
*  this project is closed and redistribution of this code is
*  prohibited. 
*  
*  Contact: http://www.groupme.net
*
*  This copyright notice MUST APPEAR in all copies of the file!
***************************************************************/
package net.groupme.analysis;

import java.io.File;
import java.io.FileWriter;
import java.sql.Date;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import net.groupme.analysis.model.SearchAndClick;
import net.groupme.exceptions.InsertFailedException;
import net.groupme.logging.model.ActionIdentifier;
import net.groupme.logging.model.WeightedResourceAndSearchString;
import net.groupme.model.MODEL_CONSTANTS;
import net.groupme.model.ObjectFactory;
import net.groupme.model.Resource;
import net.groupme.model.Tag;
import net.groupme.model.dao.DATABASE_CONSTANTS;
import net.groupme.search.strategy.SearchBasedOnGroupAndTimeContext;
import net.groupme.search.strategy.SearchBasedOnGroupContextPerformanceOptimized;
import net.groupme.search.strategy.model.WeightedIdentifiableComparatorByWeight;
import net.groupme.search.strategy.model.WeightedResource;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;


/**
 * created on Oct 4, 2007
 * 
 * This class provides some methods for analyzing the search and browsing logs.
 * 
 * @author Fabian Abel, <a href="mailto:abel@l3s.de">abel@l3s.de</a>
 * @author last edited by: $Author: fabian $
 * 
 * @version $Revision: 1.8 $ $Date: 2008-01-26 20:00:59 $
 */
public class SearchAnalyzer {
	/** Logger for this class and subclasses */
	protected final static  Log logger = LogFactory.getLog(SearchAnalyzer.class);
	
	/** 
	 *  ADJUST THIS PATH ON YOUR MACHINE!!
	 *  
	 *  Full path of gnuplot application.
	 */
	private static final String GNUPLOT_APPLICATION_PATH = "/usr/bin/gnuplot";
	
	/** 
	 *  ADJUST THIS PATH ON YOUR MACHINE!!
	 *  
	 *  Full path of epstopdf application.
	 */
	private static final String EPSTOPDF_APPLICATION_PATH = "/usr/bin/epstopdf";
	
	/*
	 * Ein paar Stats:
	 	Anteil Suchen mit anschließendem Click auf mind. eine Resource: 0.11907654921020656
		Durchschnittliche Anzahl Clicks nach Suche: 0.26123936816524906
		
		Anzahl von GroupClicks: 41
		Anteil GroupClicks / ResourceClicks: 0.19069767441860466
		-> normieren mit Gruppen-Anteil
		
		Anzahl von Clicks auf Ressourcen, die nicht mit Suchstring getagged sind (NotTaggedResourceClicks): 173
		Anteil  NotTaggedResourceClicks / ResourceClicks: 0.52222
		TODO: 
		 - nur den ersten Click angucken (nur Top k angucken)
		 - durchschnittlicher Rank auf den geklickt wird
		 - #Suchoperationen, bei denen auf mind. eine Resource geklickt wurde
		
		Durchschnittliche Anzahl Tags pro Ressource (inkl. Groups): 2.6585
		Durchschnittliche Anzahl Tags bei Gruppen: 2.9821
		Durchschnittliche Anzahl Tags bei Ressourcen (ohne Gruppen): 2.5911
		
		Anzahl untagged Resources: 147
		Anzahl tagged Resources: 325
		-> 0.3114 Anteil ungetaggter Resourcen
		
		Anzahl untagged Groups: 13
		Anzahl tagged Groups: 56
		-> 0.18 Anteil ungetaggter Groups
		
		Number of Groups: 69
		Number of Resources (without groups): 403
		Anteil Groups: 0.17
	 */
	
	/*
	 * Veraendert sich das Tagging durch die Gruppen-Dimension.
	 * 
	 * TODO:
	 * - for each Tag: Suche machen und anschauen, wie groß der Anteil der ungetaggten Ressourcen ist (+ Anteil an Gruppen)
	 * - Anzahl der ungetaggten Ressourcen
	 *   -> evtl. auch nen zeitlicher Verlauf: Wie verändert sich der Anteil ungetaggter Ressourcen/Gruppen über die Zeit
	 *      mit Anzahl Gruppen/Ressourcen ins gleiche Diagramm
	 * 
	 * - durchschn. Anzahl Tags bei Ressourcen bzw. Groups
	 * - wie, wann ist das GroupMe-System gewachsen 
	 * 
	 * Select count(tas.resourceId) from (SELECT distinct resourceId from userTagGroupResource where date < '2007-08-01'group by resourceId) as tas
	 * 
	 * Entwicklung Durchschn. Anzahl Tags pro Resource:
	 * SELECT sum(tas.tags)/res.numres from ( SELECT count(tagId) as tags, max(resourceId) as maxId FROM `userTagGroupResource` WHERE date < '2007-08-01' group by resourceId order by count(tagId) DESC ) as tas, (SELECT count(id) as numres FROM `Resource` WHERE id < (SELECT  max(resourceId) as maxId FROM `userTagGroupResource` WHERE date < '2007-08-01')) as res group by res.numres
	 * 
	 * Entwicklung Anzahl Gruppen:
	 * "SELECT count(id) as numres FROM `Resource` WHERE id < (SELECT  max(resourceId) as maxId FROM `userTagGroupResource` WHERE date < '2007-10-20') AND type = 'group'"
	 */
	
	private static final long MAX_DURATION_BETWEEN_SEARCH_AND_CLICK = 3600000; 
	
	

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		Integer[] runs = {2, 4, 7, 9, 10};
		//Integer[] runs = {11};
		for (int i = 0; i < runs.length; i++) {
			int run = runs[i];
			switch (run) {
			case 1:
				listSearchOperationsAndClicks(5, 1);
				break;
			case 2:
				computeGroupMeStats();
				break;
			case 3:
				Integer[] topKs = {3, 5, 10};
				for (int j = 0; j < topKs.length; j++) {
					performSearchForEachTag(topKs[j]);
				}
				break;
			case 4:
				//For Figure: evolution of number of resources/groups/...
				computeGroupMeEvolutionAmount();
				//For Figure: evolution of average number of tag assignments per group/resource/...
				computeGroupMeEvolutionTagging();
				break;
			case 5:
				computeAvarageNumberOfSearchResultsPerformedByUsers();
				break;
			case 6:
				postPerformEachUserSearch();
				break;
			case 7:
				computeGroupMeGroupsCharacteristics();
				break;
			case 8:
				//to compute general search operation stats:
				postPerformEachUserSearch(3);
				break;
			case 9:
				//For Figure, which illustrates the evolution of the average number of untagged groups/resources
				computeGroupMeEvolutionUntagged();
				break;
			case 10:
				// to show that our groups are real multimedia groups:
				determineMediaTypesInGroups();
				break;
			case 11:
				analyzeMostPopularResources();
				break;
			default:
				break;
			}
		}
		
	}
	
	private static final String topk = "Top k";
	private static final String avgResults = "AVG Results";
	private static final String avgGroups = "AVG Groups";
	private static final String untagged = "Untagged";
	
	/**
	 * Perform each search operation that has allready performed by a user
	 * and evaluate the search result (e.g. number of resources that are not tagged with
	 * the actual search-tag, avg number of clicks on resources )
	 *
	 * This method prints the following table
	 * top k  |  percentage of groups as result  |  percentage of group clicks  |  percentage of resources/groups that are not tagged with Search-Tag (= Untagged) |  percentage of Untagged clicks
	 * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------  
	 */
	public static void postPerformEachUserSearch(){
		String table = "\\begin{table}\n \\begin{tabular}{|c|c|c|c|c|} \n \\hline \n " +
				"\\textbf{" + topk + "} & \\textbf{" + avgGroups + "} &  \\textbf{" + anteilGroupClicks + "} & \\textbf{" + untagged + "} &  \\textbf{" + anteilUntaggedClicks + "}\\\\ \n \\hline \n";
		Map<String, Double> result = null;
		Map<String, Double> resultUserClicks = null;
		Integer[] topKs = {3, 5, 10, 15, 20, 30, 50};
		for (int j = 0; j < topKs.length; j++) {
			result = postPerformEachUserSearch(topKs[j]);
			resultUserClicks = listSearchOperationsAndClicks(topKs[j], 1);
			table += result.get(topk) + " & " + result.get(avgGroups) + " & " + resultUserClicks.get(anteilGroupClicks) + " & " + result.get(untagged) + " & " + resultUserClicks.get(anteilUntaggedClicks) +" \\\\ \n \\hline \n";
			System.out.println("Top " + result.get(topk));
		}
		table += "\\end{tabular} \n \\vspace{-0.3cm}  \n \\end{table}";
		System.out.println(table);
	}
	
	
	private static final Date START_DATE = Date.valueOf("2007-07-10");
	private static final Date END_DATE = Date.valueOf("2007-12-31");
	private static final long DAY_IN_MILLIS = 86400000;
	
	private static File evolutionNumberOfResourcesAndGroups = new File("evolution-number-of-resources-and-groups.txt");
	private static File evolutionNumberOfResources = new File("evolution-number-of-resources.txt");
	private static File evolutionNumberOfGroups = new File("evolution-number-of-groups.txt");
	
	/**
	 * Evolution of Number of Resources/Groups over time.
	 */
	public static void computeGroupMeEvolutionAmount(){
		Date actualDate = new Date(START_DATE.getTime());
		Statement sqlStatement = null;
		ResultSet result = null;
		
		try {
			sqlStatement = ObjectFactory.ds.getConnection().createStatement();
			
//			Entwicklung Anzahl Ressourcen inkl. Groups:
			FileWriter fw = new FileWriter(evolutionNumberOfResourcesAndGroups);
			while(actualDate.getTime() <= END_DATE.getTime()){
				result = sqlStatement.executeQuery("SELECT count(id) as numres FROM `Resource` WHERE id < (SELECT  max(resourceId) as maxId FROM `userTagGroupResource` WHERE date < '" + actualDate + "')");
				if(result.next()){
					fw.write(actualDate.toString() + "  " +  result.getString(1) + " \n");
					logger.info(actualDate.toString() + "  " +  result.getString(1));
				}
				actualDate.setTime(actualDate.getTime() + DAY_IN_MILLIS);
			}
			fw.flush();
			fw.close();
			
			//Entwicklung Anzahl Ressourcen exkl. Groups:
			actualDate.setTime(START_DATE.getTime());
			fw = new FileWriter(evolutionNumberOfResources);
			while(actualDate.getTime() <= END_DATE.getTime()){
				result = sqlStatement.executeQuery("SELECT count(id) as numres FROM `Resource` WHERE id < (SELECT  max(resourceId) as maxId FROM `userTagGroupResource` WHERE date < '" + actualDate + "') AND type != 'group'");
				if(result.next()){
					fw.write(actualDate.toString() + "  " +  result.getString(1) + " \n");
					logger.info(actualDate.toString() + "  " +  result.getString(1));
				}
				actualDate.setTime(actualDate.getTime() + DAY_IN_MILLIS);
			}
			fw.flush();
			fw.close();
			
			//Entwicklung Anzahl Groups:
			actualDate.setTime(START_DATE.getTime());
			fw = new FileWriter(evolutionNumberOfGroups);
			while(actualDate.getTime() <= END_DATE.getTime()){
				result = sqlStatement.executeQuery("SELECT count(id) as numres FROM `Resource` WHERE id < (SELECT  max(resourceId) as maxId FROM `userTagGroupResource` WHERE date < '" + actualDate + "') AND type = 'group'");
				if(result.next()){
					fw.write(actualDate.toString() + "  " +  result.getString(1) + " \n");
					logger.info(actualDate.toString() + "  " +  result.getString(1));
				}
				actualDate.setTime(actualDate.getTime() + DAY_IN_MILLIS);
			}
			fw.flush();
			fw.close();
			
			//Gnuplot:
			Runtime.getRuntime().exec(GNUPLOT_APPLICATION_PATH + " groupme-evolution-number.gnuplot");
			Runtime.getRuntime().exec(EPSTOPDF_APPLICATION_PATH + " groupme-evolution-number.eps");
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	private static File evolutionAvgNumberOfTags = new File("evolution-avg-number-of-tag-assignments.txt");
	private static File evolutionAvgNumberOfTagsPerResource = new File("evolution-avg-number-of-tag-assignments-per-resource.txt");
	private static File evolutionAvgNumberOfTagsPerGroup = new File("evolution-avg-number-of-tag-assignments-per-group.txt");
	private static File evolutionUntagged = new File("evolution-untagged.txt");
	private static File evolutionUntaggedResources = new File("evolution-untagged-resources.txt");
	private static File evolutionUntaggedGroups = new File("evolution-untagged-groups.txt");
	
	/**
	 * Evolution of tagging behavior...
	 *
	 */
	public static void computeGroupMeEvolutionTagging(){
		Date actualDate = new Date(START_DATE.getTime());
		Statement sqlStatement = null;
		ResultSet result = null;
		double lastTASResult = 0.0, lastGroupTAS = 0.0;
		//overall averages:
		double overallAVGOfTagsAssignedToResourcesOrGroups = 0.0;
		double overallAVGOfTagsAssignedToResources = 0.0;
		double overallAVGOfTagsAssignedToGroups = 0.0;
		double dayIterations = 0.0; 
		try {
			sqlStatement = ObjectFactory.ds.getConnection().createStatement();
			
//			Entwicklung durchschn. Anzahl an Tags:
			FileWriter fw = new FileWriter(evolutionAvgNumberOfTags);
			while(actualDate.getTime() <= END_DATE.getTime()){
				result = sqlStatement.executeQuery("SELECT sum(tas.tags)/res.numres from ( SELECT count(tagId) as tags, max(resourceId) as maxId FROM `userTagGroupResource` WHERE date < '" + actualDate + "' group by resourceId order by count(tagId) DESC ) as tas, (SELECT count(id) as numres FROM `Resource` WHERE id < (SELECT  max(resourceId) as maxId FROM `userTagGroupResource` WHERE date < '" + actualDate + "')) as res group by res.numres");
				if(result.next()){
					lastTASResult = result.getDouble(1);
					fw.write(actualDate.toString() + "  " +  result.getString(1) + " \n");
					//logger.info(actualDate.toString() + "  " +  result.getString(1));
					overallAVGOfTagsAssignedToResourcesOrGroups += lastTASResult;
					dayIterations++;
				}
				actualDate.setTime(actualDate.getTime() + DAY_IN_MILLIS);
			}
			fw.flush();
			fw.close();
			System.out.println("TAS per Resource/Group: " + lastTASResult);
			
//			Entwicklung durchschn. Anzahl an Tags fuer Ressourcen:
			actualDate.setTime(START_DATE.getTime());
			fw = new FileWriter(evolutionAvgNumberOfTagsPerResource);
			while(actualDate.getTime() <= END_DATE.getTime()){
				result = sqlStatement.executeQuery("SELECT sum(tas.tags)/res.numres from ( SELECT count(tagId) as tags, max(resourceId) as maxId FROM `userTagGroupResource` WHERE date < '" + actualDate + "' AND resourceId in (Select id from Resource where type != 'group') group by resourceId order by count(tagId) DESC ) as tas, (SELECT count(id) as numres FROM `Resource` WHERE type != 'group' AND id < (SELECT  max(resourceId) as maxId FROM `userTagGroupResource` WHERE date < '" + actualDate + "')) as res group by res.numres");
				if(result.next()){
					lastTASResult = result.getDouble(1);
					fw.write(actualDate.toString() + "  " +  result.getString(1) + " \n");
					//logger.info(actualDate.toString() + "  " +  result.getString(1));
					overallAVGOfTagsAssignedToResources += lastTASResult;
				}
				actualDate.setTime(actualDate.getTime() + DAY_IN_MILLIS);
			}
			fw.flush();
			fw.close();
			System.out.println("TAS per Resource: " + lastTASResult);
			
//			Entwicklung durchschn. Anzahl an Tags fuer Groups:
			actualDate.setTime(START_DATE.getTime());
			fw = new FileWriter(evolutionAvgNumberOfTagsPerGroup);
			while(actualDate.getTime() <= END_DATE.getTime()){
				result = sqlStatement.executeQuery("SELECT sum(tas.tags)/res.numres from ( SELECT count(tagId) as tags, max(resourceId) as maxId FROM `userTagGroupResource` WHERE date < '" + actualDate + "' AND resourceId in (Select id from Resource where type = 'group') group by resourceId order by count(tagId) DESC ) as tas, (SELECT count(id) as numres FROM `Resource` WHERE type = 'group' AND id < (SELECT  max(resourceId) as maxId FROM `userTagGroupResource` WHERE date < '" + actualDate + "')) as res group by res.numres");
				if(result.next()){
					lastGroupTAS = result.getDouble(1);
					fw.write(actualDate.toString() + "  " +  result.getString(1) + " \n");
					//logger.info(actualDate.toString() + "  " +  result.getString(1));
					overallAVGOfTagsAssignedToGroups += lastGroupTAS;
				}
				actualDate.setTime(actualDate.getTime() + DAY_IN_MILLIS);
			}
			fw.flush();
			fw.close();
			System.out.println("TAS per Group: " + lastGroupTAS);
			
//			Gnuplot:
			Runtime.getRuntime().exec(GNUPLOT_APPLICATION_PATH + " groupme-evolution-tagging.gnuplot");
			Runtime.getRuntime().exec(EPSTOPDF_APPLICATION_PATH + " groupme-evolution-tagging.eps");
			
			//Latest TAS-number-Diff:
			System.out.println("Difference (TAS per Group - TAS per Resource) = " + (lastGroupTAS - lastTASResult));
			
			//Overall AVGs:
			overallAVGOfTagsAssignedToResourcesOrGroups = overallAVGOfTagsAssignedToResourcesOrGroups / dayIterations;
			overallAVGOfTagsAssignedToResources = overallAVGOfTagsAssignedToResources / dayIterations;
			overallAVGOfTagsAssignedToGroups = overallAVGOfTagsAssignedToGroups / dayIterations;
			System.out.println("AVG TAS per Resource/Group: " + overallAVGOfTagsAssignedToResourcesOrGroups);
			System.out.println("AVG TAS per Resource: " + overallAVGOfTagsAssignedToResources);
			System.out.println("AVG TAS per Group: " + overallAVGOfTagsAssignedToGroups);
			System.out.println("Difference (AVG TAS per Group - AVG TAS per Resource) = " + (overallAVGOfTagsAssignedToGroups - overallAVGOfTagsAssignedToResources));
			
			/* OLD AND DEPRICATED
//			Entwicklung untagged Resources and Groups:
			actualDate.setTime(START_DATE.getTime());
			fw = new FileWriter(evolutionUntagged);
			while(actualDate.getTime() <= END_DATE.getTime()){
				result = sqlStatement.executeQuery("SELECT count(id) FROM Resource WHERE id NOT in (SELECT resourceId FROM `userTagGroupResource` group by resourceId) AND id < (SELECT  max(resourceId) as maxId FROM `userTagGroupResource` WHERE date < '" + actualDate + "')");
				if(result.next()){
					fw.write(actualDate.toString() + "  " +  result.getString(1) + " \n");
					//logger.info(actualDate.toString() + "  " +  result.getString(1));
				}
				actualDate.setTime(actualDate.getTime() + DAY_IN_MILLIS);
			}
			fw.flush();
			fw.close();
			
//			Entwicklung untagged Resources:
			actualDate.setTime(START_DATE.getTime());
			fw = new FileWriter(evolutionUntaggedResources);
			while(actualDate.getTime() <= END_DATE.getTime()){
				result = sqlStatement.executeQuery("SELECT count(id) FROM Resource WHERE type != 'group' AND id NOT in (SELECT resourceId FROM `userTagGroupResource` group by resourceId) AND id < (SELECT  max(resourceId) as maxId FROM `userTagGroupResource` WHERE date < '" + actualDate + "')");
				if(result.next()){
					fw.write(actualDate.toString() + "  " +  result.getString(1) + " \n");
					//logger.info(actualDate.toString() + "  " +  result.getString(1));
				}
				actualDate.setTime(actualDate.getTime() + DAY_IN_MILLIS);
			}
			fw.flush();
			fw.close();
			
//			Entwicklung untagged Resources:
			actualDate.setTime(START_DATE.getTime());
			fw = new FileWriter(evolutionUntaggedGroups);
			while(actualDate.getTime() <= END_DATE.getTime()){
				result = sqlStatement.executeQuery("SELECT count(id) FROM Resource WHERE type = 'group' AND id NOT in (SELECT resourceId FROM `userTagGroupResource` group by resourceId) AND id < (SELECT  max(resourceId) as maxId FROM `userTagGroupResource` WHERE date < '" + actualDate + "')");
				if(result.next()){
					fw.write(actualDate.toString() + "  " +  result.getString(1) + " \n");
					//logger.info(actualDate.toString() + "  " +  result.getString(1));
				}
				actualDate.setTime(actualDate.getTime() + DAY_IN_MILLIS);
			}
			fw.flush();
			fw.close();
			
			
//			Gnuplot:
			Runtime.getRuntime().exec("/usr/bin/gnuplot groupme-evolution-untagged.gnuplot");
			*/
		}catch(Exception e){
			e.printStackTrace();
		}
	}
	
	/**
	 * Evolution of average untagged groups/resources.
	 *
	 * Warum liefert folgende Anfrage kein Ergebnis, obwohl beide Unterabfragen genau ein Ergebnis liefern??
	 * SELECT (untagged.number / allres.number) 
	 *  FROM ( SELECT count(id) as number FROM Resource 
	 *  		WHERE id NOT in (SELECT resourceId FROM `userTagGroupResource` group by resourceId) 
	 *  		AND id < (SELECT  max(resourceId) as maxId FROM `userTagGroupResource` WHERE date <= '2007-10-10') 
	 *  		) as untagged, 
	 *  	( SELECT count(id) as number FROM Resource 
	 *  		WHERE id < (SELECT max(resourceId) FROM userTagGroupResource WHERE date <= '2007-10-10') 
	 *  		) as allres
	 */
	public static void computeGroupMeEvolutionUntagged(){
		Date actualDate = new Date(START_DATE.getTime());
		Statement sqlStatement = null;
		ResultSet result = null;
		double lastTASResult = 0.0, lastGroupTAS = 0.0;
		//overall averages:
		double overallAVGOfTagsAssignedToResourcesOrGroups = 0.0;
		double overallAVGOfTagsAssignedToResources = 0.0;
		double overallAVGOfTagsAssignedToGroups = 0.0;
		double dayIterations = 0.0; 
		try {
			sqlStatement = ObjectFactory.ds.getConnection().createStatement();
			
			FileWriter fw = new FileWriter(evolutionAvgNumberOfTags);
//			Entwicklung untagged Resources and Groups:
			actualDate.setTime(START_DATE.getTime());
			fw = new FileWriter(evolutionUntagged);
			double untagged = 0.0, all = 0.0; boolean queriesExecutedSuccessful = true;
			while(actualDate.getTime() <= END_DATE.getTime()){
				result = sqlStatement.executeQuery("SELECT count(id) as number FROM Resource WHERE id NOT in (SELECT resourceId FROM `userTagGroupResource` group by resourceId) AND id < (SELECT  max(resourceId) as maxId FROM `userTagGroupResource` WHERE date <= '" + actualDate + "')");
				if(result.next()){
					untagged = result.getDouble(1);
				}else{
					queriesExecutedSuccessful = false;
				}
				result = sqlStatement.executeQuery("SELECT count(id) as number FROM Resource WHERE id < (SELECT max(resourceId) FROM userTagGroupResource WHERE date <= '" + actualDate + "')");
				if(result.next()){
					all = result.getDouble(1);
				}else{
					queriesExecutedSuccessful = false;
				}
				if(queriesExecutedSuccessful){
					fw.write(actualDate.toString() + "  " +  (untagged/all) + " \n");
				}
				
				queriesExecutedSuccessful = true;
				actualDate.setTime(actualDate.getTime() + DAY_IN_MILLIS);
			}
			fw.flush();
			fw.close();
			
//			Entwicklung untagged Resources:
			actualDate.setTime(START_DATE.getTime());
			fw = new FileWriter(evolutionUntaggedResources);
			while(actualDate.getTime() <= END_DATE.getTime()){
				result = sqlStatement.executeQuery("SELECT count(id) as number FROM Resource WHERE type != 'group' AND id NOT in (SELECT resourceId FROM `userTagGroupResource` group by resourceId) AND id < (SELECT  max(resourceId) as maxId FROM `userTagGroupResource` WHERE date <= '" + actualDate + "')");
				if(result.next()){
					untagged = result.getDouble(1);
				}else{
					queriesExecutedSuccessful = false;
				}
				result = sqlStatement.executeQuery("SELECT count(id) as number FROM Resource WHERE type != 'group' AND id < (SELECT max(resourceId) FROM userTagGroupResource WHERE date <= '" + actualDate + "')");
				if(result.next()){
					all = result.getDouble(1);
				}else{
					queriesExecutedSuccessful = false;
				}
				if(queriesExecutedSuccessful){
					fw.write(actualDate.toString() + "  " +  (untagged/all) + " \n");
				}
				
				queriesExecutedSuccessful = true;
				actualDate.setTime(actualDate.getTime() + DAY_IN_MILLIS);
			}
			fw.flush();
			fw.close();
			
//			Entwicklung untagged Groups:
			actualDate.setTime(START_DATE.getTime());
			fw = new FileWriter(evolutionUntaggedGroups);
			while(actualDate.getTime() <= END_DATE.getTime()){
				result = sqlStatement.executeQuery("SELECT count(id) as number FROM Resource WHERE type = 'group' AND id NOT in (SELECT resourceId FROM `userTagGroupResource` group by resourceId) AND id < (SELECT  max(resourceId) as maxId FROM `userTagGroupResource` WHERE date <= '" + actualDate + "')");
				if(result.next()){
					untagged = result.getDouble(1);
				}else{
					queriesExecutedSuccessful = false;
				}
				result = sqlStatement.executeQuery("SELECT count(id) as number FROM Resource WHERE type = 'group' AND id < (SELECT max(resourceId) FROM userTagGroupResource WHERE date <= '" + actualDate + "')");
				if(result.next()){
					all = result.getDouble(1);
				}else{
					queriesExecutedSuccessful = false;
				}
				if(queriesExecutedSuccessful){
					fw.write(actualDate.toString() + "  " +  (untagged/all) + " \n");
				}
				
				queriesExecutedSuccessful = true;
				actualDate.setTime(actualDate.getTime() + DAY_IN_MILLIS);
			}
			fw.flush();
			fw.close();
			
			
//			Gnuplot:
			Runtime.getRuntime().exec(GNUPLOT_APPLICATION_PATH + " groupme-evolution-untagged.gnuplot");
			Runtime.getRuntime().exec(EPSTOPDF_APPLICATION_PATH + " groupme-evolution-untagged.eps");
		}catch(Exception e){
			e.printStackTrace();
		}
	}

	private static Map<String, String> QUERIES = new HashMap<String, String>();
	static {
		QUERIES.put("1.1 Number of Resources", 
				"SELECT count(id) FROM `Resource`");
		QUERIES.put("1.2 Number of Groups", 
				"SELECT count(id) FROM `Resource` WHERE type = 'group'");
		QUERIES.put("1.3 Number of Resources (without groups)", 
				"SELECT count(id) FROM `Resource` WHERE type != 'group'");
		
		QUERIES.put("2.1 Durchschnittliche Anzahl Tags pro Ressource (inkl. Groups) (ohne ungetaggte Ressourcen)", 
				"SELECT avg(tas.tags) from ( SELECT count(tagId) as tags FROM `userTagGroupResource` group by resourceId order by count(tagId) DESC ) as tas");
		QUERIES.put("2.2 Durchschnittliche Anzahl Tags pro Ressource (inkl. Groups)", 
				"SELECT sum(tas.tags)/res.numres from ( SELECT count(tagId) as tags FROM `userTagGroupResource` group by resourceId order by count(tagId) DESC ) as tas, (SELECT count(id) as numres FROM `Resource`) as res group by res.numres");
		QUERIES.put("2.3 Durchschnittliche Anzahl Tags bei Gruppen (ohne ungetaggte Gruppen)", 
				"SELECT avg(tas.tags) from ( SELECT count(tagId) as tags FROM `userTagGroupResource` WHERE resourceId in (SELECT id FROM Resource WHERE type = 'group') group by resourceId order by count(tagId) DESC ) as tas");
		QUERIES.put("2.4 Durchschnittliche Anzahl Tags bei Gruppen", 
			"SELECT sum(tas.tags)/res.numres from ( SELECT count(tagId) as tags FROM `userTagGroupResource` WHERE resourceId in (SELECT id FROM Resource WHERE type = 'group') group by resourceId order by count(tagId) DESC ) as tas, (SELECT count(id) as numres FROM `Resource` WHERE type = 'group') as res group by res.numres");
		QUERIES.put("2.5 Durchschnittliche Anzahl Tags bei Ressourcen (ohne Gruppen) (ohne ungetaggte Ressourcen)", 
				"SELECT avg(tas.tags) from ( SELECT count(tagId) as tags FROM `userTagGroupResource` WHERE resourceId in (SELECT id FROM Resource WHERE type != 'group') group by resourceId order by count(tagId) DESC ) as tas");
		QUERIES.put("2.6 Durchschnittliche Anzahl Tags bei Ressourcen (ohne Gruppen)", 
				"SELECT sum(tas.tags)/res.numres from ( SELECT count(tagId) as tags FROM `userTagGroupResource` WHERE resourceId in (SELECT id FROM Resource WHERE type != 'group') group by resourceId order by count(tagId) DESC ) as tas, (SELECT count(id) as numres FROM `Resource` WHERE type != 'group') as res group by res.numres");

		QUERIES.put("3.1 Anzahl Tag Assignments", 
				"SELECT count(*) FROM `userTagGroupResource`");
		QUERIES.put("3.2 Anzahl Tag Assignments bei Groups", 
				"SELECT count(*)  FROM `userTagGroupResource` WHERE resourceId in (SELECT id FROM Resource WHERE type = 'group')");
		QUERIES.put("3.3 Anzahl Tag Assignments bei Ressourcen", 
				"SELECT count(*)  FROM `userTagGroupResource` WHERE resourceId in (SELECT id FROM Resource WHERE type != 'group')");

		QUERIES.put("4.1 Anzahl untagged Resources (inkl. Groups)", 
				"SELECT count(id) FROM Resource WHERE id NOT in (SELECT resourceId FROM `userTagGroupResource` group by resourceId)");
		QUERIES.put("4.2 Anzahl tagged Resources (inkl. Groups)", 
				"SELECT count(id) FROM Resource WHERE id in (SELECT resourceId FROM `userTagGroupResource` group by resourceId)");
		QUERIES.put("4.3 Anzahl untagged Groups", 
				"SELECT count(id) FROM Resource WHERE id NOT in (SELECT resourceId FROM `userTagGroupResource` group by resourceId) AND type = 'group'");
		QUERIES.put("4.4 Anzahl tagged Groups", 
				"SELECT count(id) FROM Resource WHERE id in (SELECT resourceId FROM `userTagGroupResource` group by resourceId) AND type = 'group'");
		QUERIES.put("4.5 Anzahl untagged Resources (without Groups)", 
				"SELECT count(id) FROM Resource WHERE id NOT in (SELECT resourceId FROM `userTagGroupResource` group by resourceId) AND type != 'group'");
		QUERIES.put("4.6 Anzahl tagged Resources (without Groups)", 
				"SELECT count(id) FROM Resource WHERE id in (SELECT resourceId FROM `userTagGroupResource` group by resourceId) AND type != 'group'");
		
	}

	/**
	 * GroupMe! stats like average number of tags per resource, ...
	 */
	public static void computeGroupMeStats(){
		Statement sqlStatement = null;
		ResultSet result = null;
		try {
			sqlStatement = ObjectFactory.ds.getConnection().createStatement();
			
			List<String> orderedKeys = new ArrayList<String>(QUERIES.keySet());
			Collections.sort(orderedKeys);
			for (String key : orderedKeys) {
				result = sqlStatement.executeQuery(QUERIES.get(key));
				System.out.println(key + ": " + (result.next() ? result.getString(1) : " NO RESULT!!"));				
			}
		} catch (SQLException e) {
			e.printStackTrace();
			logger.error("Have you selected the right database? (see preference file)", e);
		}
	}

	public static void computeAvarageNumberOfSearchResultsPerformedByUsers(){
		List<SearchAndClick> searchAndClickActions = getSearchStringAndResourceClick(false);
		int overallNumberOfSearchResults = 0;
		int overallNumberOfGroupSearchResults = 0;
		for (SearchAndClick sac : searchAndClickActions) {
			SearchBasedOnGroupContextPerformanceOptimized search = new SearchBasedOnGroupContextPerformanceOptimized();
			List<Tag> tagObj = new ArrayList<Tag>();
			Tag tag = null;
			try {
				tag = ObjectFactory.getTagForKeyword(sac.getSearchString().replaceAll("\"", ""));
			} catch (InsertFailedException e) {
				e.printStackTrace();
			}
			tagObj.add(tag);
			List<Resource> resources = search.searchForResources(tagObj, false, false);
			overallNumberOfSearchResults += resources.size();
			for(Resource res: resources){
				if(MODEL_CONSTANTS.RESOURCE_TYPE_GROUP.equals(res.getType())){
					overallNumberOfGroupSearchResults++;
				}
			}
		}
		double numberOfSearchOperations = new Double(searchAndClickActions.size());
		System.out.println("Number of search operations performed by users: " + numberOfSearchOperations);
		System.out.println("AVG number of results after a user performed search: " + (new Double(overallNumberOfSearchResults) / numberOfSearchOperations) );
		System.out.println("AVG number of resource-results after a user performed search: " + (new Double(overallNumberOfSearchResults - overallNumberOfGroupSearchResults) / numberOfSearchOperations) );
		System.out.println("AVG number of group-results after a user performed search: " + (new Double(overallNumberOfGroupSearchResults) / numberOfSearchOperations) );
		
	}
	
	private static final String anteilGroupClicks = "Group Clicks";
	private static final String anteilUntaggedClicks = "Unttaged Clicks";
	
	//private static List<SearchAndClick> searchAndClicks = null;
	
	/**
	 * Search- and Clickbehavior...
	 */
	public static Map<String, Double> listSearchOperationsAndClicks(int topk, int numberOfFirstClicks){
		int clicksAfterSearch = 0;
		int clicks = 0;
		int numberOfGroupClicks = 0;
		int numberOfResourcesThatWereNotTaggedWithSearchKeyword = 0;
			List<SearchAndClick>	searchAndClicks = getSearchStringAndResourceClick(true);
		
		for (SearchAndClick sac : searchAndClicks) {
			
			//Clean SaC -> remove resources that do not occur in the search result
			//eigentlich muesste man die Suchergebnisliste so bestimmen, wie sie
			//zum Zeitpunkt der Suche zustande gekommen waere... aber egal.
			cleanListOfVisitedResourcesAfterSearch(sac, topk, numberOfFirstClicks);
			
			//clicks zaehlen:
			if(sac.getResourcesVisited().size() > 0){
				clicksAfterSearch += 1;
				clicks += sac.getResourcesVisited().size();
			}
			
			//ressourcen, auf die geclickt wurde genauer anschauen:
			for(String resId: sac.getResourcesVisited()){
				Resource resource = ObjectFactory.getResource(resId);
				if(MODEL_CONSTANTS.RESOURCE_TYPE_GROUP.equals(resource.getType())){
					numberOfGroupClicks++;
				}
				boolean tagFound = false;
				String tags = "";
				for(Tag tag: resource.getTags()){
					tags += " " + tag.getKeyword();
					if(sac.getSearchString().equals(tag.getKeyword())){
						tagFound = true;
					}
				}
				//System.out.println("SearchString: " + sac.getSearchString() + " vs. tags: " + tags + " -> " + (tagFound ? " tagged with SearchString" : " not tagged with SearchString"));
				if(!tagFound){
					numberOfResourcesThatWereNotTaggedWithSearchKeyword++;
				}
			}
		}
		
		System.out.println("Anteil Suchen mit anschließendem Click auf mind. eine Resource: " + (new Double(clicksAfterSearch) / new Double(searchAndClicks.size())));
		//System.out.println("Durchschnittliche Anzahl Clicks nach Suche: " + (new Double(clicks) / new Double(searchAndClicks.size())));
		
		System.out.println("Anzahl von GroupClicks: " + numberOfGroupClicks);
		System.out.println("Anteil GroupClicks / ResourceClicks: " + (new Double(numberOfGroupClicks) / new Double(clicks)));
		
		System.out.println("Anzahl von Clicks auf Ressourcen, die nicht mit Suchstring getagged sind (NotTaggedResourceClicks): " + numberOfResourcesThatWereNotTaggedWithSearchKeyword);
		System.out.println("Anteil  NotTaggedResourceClicks / ResourceClicks: " + (new Double(numberOfResourcesThatWereNotTaggedWithSearchKeyword) / new Double(clicks)));
		
		Map<String, Double> result = new HashMap<String, Double>();
		result.put(SearchAnalyzer.topk, new Double(topk));
		result.put(anteilGroupClicks, (new Double(numberOfGroupClicks) / new Double(clicks)));
		result.put(anteilUntaggedClicks, (new Double(numberOfResourcesThatWereNotTaggedWithSearchKeyword) / new Double(clicks)));
		return result;
	}
	
	/**
	 * Returns Search-Click-User-Actions...
	 * @param onlyWithClicks if this parameter is <code>true</code> then only Search-Click-User-Actions, in which the user clicked at least on one resource, are returned.
	 * @return Search-Click-User-Actions...
	 */
	public static List<SearchAndClick> getSearchStringAndResourceClick(boolean onlyWithClicks){
		Statement sqlStatement = null;
		ResultSet loggingTable = null;
		List<SearchAndClick> searchAndClicks = new ArrayList<SearchAndClick>();
		try {
			sqlStatement = ObjectFactory.ds.getConnection().createStatement();
			loggingTable = sqlStatement.executeQuery("SELECT * FROM " + DATABASE_CONSTANTS.TABLE_LOG_EVENT_NAVIGATION);
			int actualRow = 0;
			while(loggingTable.next()){
				actualRow = loggingTable.getRow();
				
				//create a new SearchAndClick instance from the current row if action is not a resourceClick
				if(!ActionIdentifier.RESOURCE_CLICK.getIdentifierValue().equals(loggingTable.getString(DATABASE_CONSTANTS.LOG_EVENT_NAVIGATION_ACTION))){
					SearchAndClick sac = new SearchAndClick(
							loggingTable.getString(DATABASE_CONSTANTS.LOG_EVENT_NAVIGATION_SEARCH_STRING), 
							(ActionIdentifier.FREE_SEARCH.getIdentifierValue().equals(loggingTable.getString(DATABASE_CONSTANTS.LOG_EVENT_NAVIGATION_ACTION)) ? ActionIdentifier.FREE_SEARCH : ActionIdentifier.TAG_SEARCH),
							loggingTable.getString(DATABASE_CONSTANTS.LOG_EVENT_NAVIGATION_USER_ID),
							loggingTable.getString(DATABASE_CONSTANTS.LOG_EVENT_NAVIGATION_SESSION_ID),
							loggingTable.getDate(DATABASE_CONSTANTS.LOG_EVENT_NAVIGATION_TIMESTAMP));
					long timestamp = sac.getDate().getTime();
					int row = 1; boolean proceed = true;
					while(loggingTable.absolute(actualRow + row) && ((loggingTable.getDate(DATABASE_CONSTANTS.LOG_EVENT_NAVIGATION_TIMESTAMP).getTime() - timestamp) < MAX_DURATION_BETWEEN_SEARCH_AND_CLICK) && proceed){
						if(ActionIdentifier.RESOURCE_CLICK.getIdentifierValue().equals(loggingTable.getString(DATABASE_CONSTANTS.LOG_EVENT_NAVIGATION_ACTION))
								&& sac.getSessionId().equals(loggingTable.getString(DATABASE_CONSTANTS.LOG_EVENT_NAVIGATION_SESSION_ID))){
							sac.addResourceVisited(loggingTable.getString(DATABASE_CONSTANTS.LOG_EVENT_NAVIGATION_RESOURCE_ID));
						}else if(sac.getSessionId().equals(loggingTable.getString(DATABASE_CONSTANTS.LOG_EVENT_NAVIGATION_SESSION_ID)) 
								&& !ActionIdentifier.RESOURCE_CLICK.getIdentifierValue().equals(loggingTable.getString(DATABASE_CONSTANTS.LOG_EVENT_NAVIGATION_ACTION))){
							proceed = false;
						}
						row++;
					}
					if(!onlyWithClicks || !sac.getResourcesVisited().isEmpty()){
						searchAndClicks.add(sac);
					}
					//debug
//					if(searchAndClicks.size() > 100){
//						return searchAndClicks;
//					}
					loggingTable.absolute(actualRow);
				}
			}
		} catch (SQLException e) {
			e.printStackTrace();
		}
		return searchAndClicks;
	}
	
	/**
	 * Clean SaC -> remove resources that do not occur in the search result
	 * 
	 * eigentlich muesste man die Suchergebnisliste so bestimmen, wie sie
	 * zum Zeitpunkt der Suche zustande gekommen waere... aber egal.
	 * 
	 * @param sac SearchAndClick instance that should be cleaned
	 */
	private static void cleanListOfVisitedResourcesAfterSearch(SearchAndClick sac, int onlyConsiderTopK, int maxNumberOfFirstClicks){
		SearchBasedOnGroupContextPerformanceOptimized search = new SearchBasedOnGroupContextPerformanceOptimized();
		List<Tag> tagObj = new ArrayList<Tag>();
		Tag tag = null;
		try {
			tag = ObjectFactory.getTagForKeyword(sac.getSearchString().replaceAll("\"", ""));
		} catch (InsertFailedException e) {
			e.printStackTrace();
		}
		tagObj.add(tag);
		logger.info("tag to clean: " + tag.getKeyword());
		
		List<Resource> resources = search.searchForResources(tagObj, false, false);
		List<String> resourceIdsInSearchResult = new ArrayList<String>(); boolean found = false;
		Iterator<String> idIterator = sac.getResourcesVisited().iterator();
		String resId = null; int clicks = 0;
		while(idIterator.hasNext() && clicks < maxNumberOfFirstClicks){
			resId = idIterator.next();
			Iterator<Resource> resourceIterator = resources.iterator(); 
			Resource resource = null; int rank = 1;
			while(resourceIterator.hasNext() && rank <= onlyConsiderTopK) {
				resource = resourceIterator.next();
				if(resource.getId().equals(resId)){
					found = true;
				}
				rank++;
			}
			if(found){
				resourceIdsInSearchResult.add(resId);
				clicks++;
			}
			
		}
		logger.info("Anzahl entfernter Ressourcen: " + (sac.getResourcesVisited().size() - resourceIdsInSearchResult.size()));
		sac.setResourcesVisited(resourceIdsInSearchResult);
	}
	
	/*
	 * Analysis of Top 3
Durchschn. Anteil an Ressourcen in der Top 3, die nicht mit Search-Tag getaggt sind (ungewichtet): 0.04977905010163073
Durchschn. Anteil an Ressourcen in der Top 3, die nicht mit Search-Tag getaggt sind (entsprechend Anzahl Suchergebnisse gewichtet): 0.08134805345729228
Durchschn. Anzahl der Searchresults: 3.7010752688172044
Durchschn. Anzahl an Groups in Searchresults: 0.07030796048808832
Analysis of Top 5
Durchschn. Anteil an Ressourcen in der Top 5, die nicht mit Search-Tag getaggt sind (ungewichtet): 0.10431733690565466
Durchschn. Anteil an Ressourcen in der Top 5, die nicht mit Search-Tag getaggt sind (entsprechend Anzahl Suchergebnisse gewichtet): 0.19291109819872168
Durchschn. Anzahl der Searchresults: 3.7010752688172044
Durchschn. Anzahl an Groups in Searchresults: 0.08483439860546194
Analysis of Top 10
Durchschn. Anteil an Ressourcen in der Top 10, die nicht mit Search-Tag getaggt sind (ungewichtet): 0.18665344138638404
Durchschn. Anteil an Ressourcen in der Top 10, die nicht mit Search-Tag getaggt sind (entsprechend Anzahl Suchergebnisse gewichtet): 0.40383497966298665
Durchschn. Anzahl der Searchresults: 3.7010752688172044
Durchschn. Anzahl an Groups in Searchresults: 0.09355026147588612
Analysis of Top 15
Durchschn. Anteil an Ressourcen in der Top 15, die nicht mit Search-Tag getaggt sind (ungewichtet): 0.1957899187615642
Durchschn. Anteil an Ressourcen in der Top 15, die nicht mit Search-Tag getaggt sind (entsprechend Anzahl Suchergebnisse gewichtet): 0.4439279488669378
Durchschn. Anzahl der Searchresults: 3.7010752688172044
Durchschn. Anzahl an Groups in Searchresults: 0.09819872167344568
Analysis of Top 20
Durchschn. Anteil an Ressourcen in der Top 20, die nicht mit Search-Tag getaggt sind (ungewichtet): 0.19924942226966752
Durchschn. Anteil an Ressourcen in der Top 20, die nicht mit Search-Tag getaggt sind (entsprechend Anzahl Suchergebnisse gewichtet): 0.46484601975595585
Durchschn. Anzahl der Searchresults: 3.7010752688172044
Durchschn. Anzahl an Groups in Searchresults: 0.09936083672283556
Analysis of Top 30
Durchschn. Anteil an Ressourcen in der Top 30, die nicht mit Search-Tag getaggt sind (ungewichtet): 0.20158843365976037
Durchschn. Anteil an Ressourcen in der Top 30, die nicht mit Search-Tag getaggt sind (entsprechend Anzahl Suchergebnisse gewichtet): 0.48576409064497383
Durchschn. Anzahl der Searchresults: 3.7010752688172044
Durchschn. Anzahl an Groups in Searchresults: 0.09936083672283556
Analysis of Top 50
Durchschn. Anteil an Ressourcen in der Top 50, die nicht mit Search-Tag getaggt sind (ungewichtet): 0.20284763003780226
Durchschn. Anteil an Ressourcen in der Top 50, die nicht mit Search-Tag getaggt sind (entsprechend Anzahl Suchergebnisse gewichtet): 0.5008715862870424
Durchschn. Anzahl der Searchresults: 3.7010752688172044
Durchschn. Anzahl an Groups in Searchresults: 0.09936083672283556
Analysis of Top 100
Durchschn. Anteil an Ressourcen in der Top 100, die nicht mit Search-Tag getaggt sind (ungewichtet): 0.20284763003780226
Durchschn. Anteil an Ressourcen in der Top 100, die nicht mit Search-Tag getaggt sind (entsprechend Anzahl Suchergebnisse gewichtet): 0.5008715862870424
Durchschn. Anzahl der Searchresults: 3.7010752688172044
Durchschn. Anzahl an Groups in Searchresults: 0.09936083672283556
	 */
	/** performSearchForEachTag() soll nur die TOP_K Ressourcen betrachten */ 
	private static final int TOP_K = 20;
	/**
	 * Perform a search for each tag that has been used in GroupMe! to tag resurces
	 * and evaluate the search result (e.g. number of resources that are not tagged with
	 * the actual search-tag)
	 *
	 */
	public static void performSearchForEachTag(int topK){
		Statement sqlStatement = null;
		ResultSet loggingTable = null;
		SearchBasedOnGroupContextPerformanceOptimized search = new SearchBasedOnGroupContextPerformanceOptimized();
		int numberOfTags = 0;
		int overallNumberOfSearchResults = 0;
		int overallNumberOfGroupSearchResults = 0;
		int overallNumberOfUntaggedSearchResults = 0;
		int actualNumberOfSearchResults = 0;
		int actualNumberOfUntaggedSearchResults = 0;
		double avgNumberOfResourcesThatDoNotHaveTheSearchTag = 0.0;
		
		try {
			sqlStatement = ObjectFactory.ds.getConnection().createStatement();
			loggingTable = sqlStatement.executeQuery("SELECT keyword  from Tag where id in (select tagId from userTagGroupResource)");
			Tag tag = null;
			List<Tag> tags = null;
			List<Resource> searchResult = null;
			Resource actualResource = null;
			while(loggingTable.next()){
				actualNumberOfUntaggedSearchResults = 0;
				tag = ObjectFactory.getTag(loggingTable.getString(1));
				tags = new ArrayList<Tag>();
				tags.add(tag);
				
				//perform search:
				searchResult = search.searchForResources(tags, false, false);
				actualNumberOfSearchResults = (searchResult.size() > topK ? topK : searchResult.size());
				overallNumberOfSearchResults += actualNumberOfSearchResults;
				for (int i = 0; i < searchResult.size() && i < topK; i++) {
					actualResource = searchResult.get(i);
					if(!actualResource.getTags().contains(tag)){
						actualNumberOfUntaggedSearchResults++;
						overallNumberOfUntaggedSearchResults++;
					}
					if(MODEL_CONSTANTS.RESOURCE_TYPE_GROUP.equals(actualResource.getType())){
						overallNumberOfGroupSearchResults++;
					}
				}
//				logger.info("Actual tag: " + tag.getKeyword());
//				logger.info("Aktuelle Anzahl ungetaggter Ressourcen: " + actualNumberOfUntaggedSearchResults);
//				logger.info("Aktueller Anteil ungetaggter Ressourcen: " + (new Double(actualNumberOfUntaggedSearchResults)/new Double(actualNumberOfSearchResults)));
				avgNumberOfResourcesThatDoNotHaveTheSearchTag = new Double(avgNumberOfResourcesThatDoNotHaveTheSearchTag * numberOfTags + (new Double(actualNumberOfUntaggedSearchResults)/new Double(actualNumberOfSearchResults)))/new Double(numberOfTags + 1);
				numberOfTags++; 
			}
			System.out.println("Analysis of Top " + topK);
			System.out.println("Durchschn. Anteil an Ressourcen in der Top " + topK + ", die nicht mit Search-Tag getaggt sind (ungewichtet): " + avgNumberOfResourcesThatDoNotHaveTheSearchTag);
			System.out.println("Durchschn. Anteil an Ressourcen in der Top " + topK + ", die nicht mit Search-Tag getaggt sind (entsprechend Anzahl Suchergebnisse gewichtet): " + (new Double(overallNumberOfUntaggedSearchResults) / new Double(overallNumberOfSearchResults)));
			System.out.println("Durchschn. Anzahl der Searchresults: " + (new Double(overallNumberOfSearchResults) / new Double(numberOfTags)));
			System.out.println("Durchschn. Anzahl an Groups in Searchresults: " + (new Double(overallNumberOfGroupSearchResults) / new Double(overallNumberOfSearchResults)));
			
		} catch (SQLException e) {
			e.printStackTrace();
			logger.error("Have you selected the right database? (see preference file)", e);
		}
	}
	
	/**
	 * Perform each search operation that has allready performed by a user
	 * and evaluate the search result (e.g. number of resources that are not tagged with
	 * the actual search-tag)
	 *
	 */
	public static Map<String, Double> postPerformEachUserSearch(int topK){
		//SearchBasedOnGroupContextPerformanceOptimized search = new SearchBasedOnGroupContextPerformanceOptimized();
		SearchBasedOnGroupAndTimeContext search = new SearchBasedOnGroupAndTimeContext();
		int numberOfTags = 0;
		int overallNumberOfSearchResults = 0;
		int overallNumberOfSearchResultsAuchAusserhalbTopK = 0;
		//int overallNumberOfResourceSearchResultsAuchAusserhalbTopK = 0;
		int overallNumberOfGroupSearchResultsAuchAusserhalbTopK = 0;
		int overallNumberOfUntaggedSearchResultsAuchAusserhalbTopK = 0;
		int overallNumberOfGroupSearchResults = 0;
		int overallNumberOfUntaggedSearchResults = 0;
		int actualNumberOfSearchResults = 0;
		int actualNumberOfUntaggedSearchResults = 0;
		double avgNumberOfResourcesThatDoNotHaveTheSearchTag = 0.0;
		
		List<SearchAndClick> searchAndClicks = getSearchStringAndResourceClick(true);

		System.out.println("Number of Search-And-Click-User-Actions: " + searchAndClicks.size());
		int searchCount = 0;
		for (SearchAndClick sac : searchAndClicks) {
			List<Tag> tagObj = new ArrayList<Tag>();
			Tag tag = null;
			try {
				tag = ObjectFactory.getTagForKeyword(sac.getSearchString()
						.replaceAll("\"", ""));
			} catch (InsertFailedException e) {
				e.printStackTrace();
			}
			tagObj.add(tag);
			List<Resource> searchResult = search.searchForResources(tagObj,
					false, false, sac.getDate());
			Resource actualResource = null;
			overallNumberOfSearchResultsAuchAusserhalbTopK += searchResult.size();
			actualNumberOfSearchResults = (searchResult.size() > topK ? topK : searchResult.size());
			overallNumberOfSearchResults += actualNumberOfSearchResults;
			for (int i = 0; i < searchResult.size(); i++) {
				actualResource = searchResult.get(i);
				if (!actualResource.getTags().contains(tag)) {
					if(i < topK){
						actualNumberOfUntaggedSearchResults++;
						overallNumberOfUntaggedSearchResults++;
					}
					overallNumberOfUntaggedSearchResultsAuchAusserhalbTopK++;
				}
				if (MODEL_CONSTANTS.RESOURCE_TYPE_GROUP.equals(actualResource
						.getType())) {
					if(i < topK){
						overallNumberOfGroupSearchResults++;
					}
					overallNumberOfGroupSearchResultsAuchAusserhalbTopK++;
				}
				
			}
			avgNumberOfResourcesThatDoNotHaveTheSearchTag = new Double(
					avgNumberOfResourcesThatDoNotHaveTheSearchTag
							* numberOfTags
							+ (new Double(actualNumberOfUntaggedSearchResults) / new Double(
									actualNumberOfSearchResults)))
					/ new Double(numberOfTags + 1);
			numberOfTags++;
			
			searchCount++;
			//System.out.println("Run " + searchCount);
		}
		Map<String, Double> result = new HashMap<String, Double>();
		result.put(topk, new Double(topK));
		result.put(avgResults, (new Double(overallNumberOfSearchResults) / new Double(numberOfTags)));
		result.put(avgGroups, (new Double(overallNumberOfGroupSearchResults) / new Double(overallNumberOfSearchResults)));
		result.put(untagged, (new Double(overallNumberOfUntaggedSearchResults) / new Double(overallNumberOfSearchResults)));
		
		System.out.println("Overall Number of Search Operations: " + searchAndClicks.size());
		System.out.println("Average number of search results: " + (new Double(overallNumberOfSearchResultsAuchAusserhalbTopK) / new Double(searchAndClicks.size())));
		System.out.println("Average number of resource search results: " + (new Double((overallNumberOfSearchResultsAuchAusserhalbTopK - overallNumberOfGroupSearchResultsAuchAusserhalbTopK)) / new Double(searchAndClicks.size())));
		System.out.println("Average number of group search results: " + (new Double(overallNumberOfGroupSearchResultsAuchAusserhalbTopK) / new Double(searchAndClicks.size())));
		System.out.println("Average number of Not-With-The-Search-Tag-tagged search results: " + (new Double(overallNumberOfUntaggedSearchResultsAuchAusserhalbTopK) / new Double(searchAndClicks.size())));
		
		return result;
	}

	private static File groupChracteristicsGroupSize = new File("group-chracteristics-size.txt");
	/**
	 * Group Characteristics like number of resources...
	 */
	public static void computeGroupMeGroupsCharacteristics(){
		Statement sqlStatement = null;
		ResultSet result = null;
		
		try {
			sqlStatement = ObjectFactory.ds.getConnection().createStatement();
			
//			Entwicklung Anzahl Ressourcen inkl. Groups:
			FileWriter fw = new FileWriter(groupChracteristicsGroupSize);
			result = sqlStatement.executeQuery("SELECT count(*) FROM Resource WHERE type = 'group' AND id not in ( SELECT groupId FROM `groupResource` WHERE 1 group by groupId)");
			if(result.next()){
				fw.write("0  " +  result.getString(1) + " \n");
			}
			result = sqlStatement.executeQuery("SELECT count(groups.numRes), groups.numRes FROM ( SELECT groupId, count(resourceId) as numRes FROM `groupResource` WHERE 1 group by groupId) as groups group by groups.numRes order by groups.numRes ASC");
			while(result.next()){
				fw.write(result.getString(2) + "  " +  result.getString(1) + " \n");
			}
			fw.flush();
			fw.close();
			
			//Gnuplot:
			Runtime.getRuntime().exec(GNUPLOT_APPLICATION_PATH + " characteristics.gnuplot");
			Runtime.getRuntime().exec(EPSTOPDF_APPLICATION_PATH + " groupme-characteristics-size.eps");
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	public static void determineMediaTypesInGroups(){
		Statement sqlStatement = null;
		ResultSet result = null;
		String queryTypeAndAmount = "SELECT res.type, count(res.type) FROM Resource as res, groupResource as gr WHERE gr.resourceId = res.id group by res.type";
		String numberOfResourceGroupAssignmentsQuery = "SELECT count(*) FROM groupResource";
		try {
			sqlStatement = ObjectFactory.ds.getConnection().createStatement();
			result = sqlStatement.executeQuery(numberOfResourceGroupAssignmentsQuery);
			double numberOfResourceGroupAssignments = 0.0;
			if(result.next()){
				numberOfResourceGroupAssignments = result.getDouble(1);
			}
			Map<String, Double> typeAmountMapping = new HashMap<String, Double>();
			result = sqlStatement.executeQuery(queryTypeAndAmount);
			while(result.next()){
				typeAmountMapping.put(result.getString(1), result.getDouble(2));
			}
			String table = "\\begin{table}\n \\begin{tabular}{c|c} \n " +
				"\\textbf{Type} & \\textbf{AVG Occurences} \\\\ \n " ;
			String[] order = {MODEL_CONSTANTS.RESOURCE_TYPE_IMAGE, MODEL_CONSTANTS.RESOURCE_TYPE_SHOCKWAVE, MODEL_CONSTANTS.RESOURCE_TYPE_RSS, MODEL_CONSTANTS.RESOURCE_TYPE_GROUP, MODEL_CONSTANTS.RESOURCE_TYPE_WEBSITE};
			String[] titles = {"images", "videos", "rss feeds", "groups", "other web resources" };
			double avg = 0.0;
			for (int i = 0; i < titles.length; i++) {
				avg = typeAmountMapping.get(order[i]) / numberOfResourceGroupAssignments;
				table += "\\hline \n " + titles[i] + " & " + ( round(100.0 * avg, 2) ) + "\\% \\\\ \n ";
			}
			table += "\\end{tabular} \n \\vspace{-0.3cm}  \n \\end{table}";
			System.out.println(table);
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	
	
	/* NEW STUFF FOR THE ICWSM PAPER */
	
	public static final void analyzeMostPopularResources(){
		List<WeightedResourceAndSearchString> ranking = getMostPopularResources(20);
		int count = 1;
		String table = "\\begin{table}\n \\begin{tabular}{|c|c|c|c|} \n " +
						"\\hline \n \\textbf{Rank} & \\textbf{Resource} & \\textbf{Type} & \\textbf{SOP} \\\\ \n \\hline " ;
		
		for (WeightedResourceAndSearchString wrs : ranking) {
			System.out.println(count + ". " + wrs);
			table += "   " + count + ". & " + wrs.toLatexString() + " \\\\ \n \\hline \n"; 
			count++;
		}
		table += "\\end{tabular} \n \\vspace{-0.3cm}  \n \\end{table}";
		System.out.println(table);
	}
	
	
	
	/**
	 * Returns an ordered list of the most popular Resources.
	 * Most Popular Resource = Resource, on which most users clicked after a search operation.
	 * @param topK the top k resources to consider 
	 * @return an ordered list of the most popular Resources
	 */
	private static final List<WeightedResourceAndSearchString> getMostPopularResources(int topK){
		List<WeightedResourceAndSearchString> ranking = new ArrayList<WeightedResourceAndSearchString>();
		List<SearchAndClick> sacs = getSearchStringAndResourceClick(true);
		
		WeightedResourceAndSearchString actualWeightedResource = null;
		int index = -1;
		for (SearchAndClick sac : sacs) {
			for(String resourceId: sac.getResourcesVisited()){
				actualWeightedResource = new WeightedResourceAndSearchString(ObjectFactory.getResource(resourceId), 1.0);
				index = ranking.indexOf(actualWeightedResource);
				if(index >= 0){
					ranking.get(index).weight += 1;
					ranking.get(index).searchStrings.add(sac.getSearchString());
				}else{
					actualWeightedResource.searchStrings.add(sac.getSearchString());
					ranking.add(actualWeightedResource);
				}
			}
		}
		
		Collections.sort(ranking, new WeightedIdentifiableComparatorByWeight());
		if(ranking.size() >= topK){
			return ranking.subList(0, topK);
		}
		return ranking;
	}
	
	private static Double round(double number, int precision){
		double tenPow = Math.pow(10, precision);
		return Math.floor(number) + (Math.floor(tenPow * (number - Math.floor(number)))/tenPow); 
	}
	
}
